{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AYV_dMVDxyc2"
   },
   "source": [
    "[![Github](https://img.shields.io/github/stars/labmlai/annotated_deep_learning_paper_implementations?style=social)](https://github.com/labmlai/annotated_deep_learning_paper_implementations)\n",
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/transformers/feedback/experiment.ipynb)                    \n",
    "\n",
    "## Feedback Transformer\n",
    "\n",
    "This is an experiment training Shakespeare dataset with Feedback Transformer."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AahG_i2y5tY9"
   },
   "source": [
    "Install the `labml-nn` package"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZCzmCrAIVg0L",
    "outputId": "aa1fe63d-1755-4394-dcdf-9897ac6c1ee4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting labml-nn\n",
      "\u001B[?25l  Downloading https://files.pythonhosted.org/packages/69/4d/ab1bc1578d83bae243118abe5c89bc9995d0195ee1d03960cae42ff39879/labml_nn-0.4.77-py3-none-any.whl (103kB)\n",
      "\u001B[K     |████████████████████████████████| 112kB 13.7MB/s \n",
      "\u001B[?25hCollecting labml>=0.4.86\n",
      "\u001B[?25l  Downloading https://files.pythonhosted.org/packages/a7/d3/f8708934e0062e6403faa2a36d97e1677097740c94f90fd7c04ea986d7cf/labml-0.4.89-py3-none-any.whl (97kB)\n",
      "\u001B[K     |████████████████████████████████| 102kB 11.1MB/s \n",
      "\u001B[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from labml-nn) (1.19.4)\n",
      "Collecting einops\n",
      "  Downloading https://files.pythonhosted.org/packages/5d/a0/9935e030634bf60ecd572c775f64ace82ceddf2f504a5fd3902438f07090/einops-0.3.0-py2.py3-none-any.whl\n",
      "Collecting labml-helpers>=0.4.72\n",
      "  Downloading https://files.pythonhosted.org/packages/ec/58/2b7dcfde4565134ad97cdfe96ad7070fef95c37be2cbc066b608c9ae5c1d/labml_helpers-0.4.72-py3-none-any.whl\n",
      "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from labml-nn) (1.7.0+cu101)\n",
      "Collecting pyyaml>=5.3.1\n",
      "\u001B[?25l  Downloading https://files.pythonhosted.org/packages/64/c2/b80047c7ac2478f9501676c988a5411ed5572f35d1beff9cae07d321512c/PyYAML-5.3.1.tar.gz (269kB)\n",
      "\u001B[K     |████████████████████████████████| 276kB 40.5MB/s \n",
      "\u001B[?25hCollecting gitpython\n",
      "\u001B[?25l  Downloading https://files.pythonhosted.org/packages/d7/cb/ec98155c501b68dcb11314c7992cd3df6dce193fd763084338a117967d53/GitPython-3.1.12-py3-none-any.whl (159kB)\n",
      "\u001B[K     |████████████████████████████████| 163kB 50.4MB/s \n",
      "\u001B[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from torch->labml-nn) (3.7.4.3)\n",
      "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from torch->labml-nn) (0.8)\n",
      "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch->labml-nn) (0.16.0)\n",
      "Collecting gitdb<5,>=4.0.1\n",
      "\u001B[?25l  Downloading https://files.pythonhosted.org/packages/48/11/d1800bca0a3bae820b84b7d813ad1eff15a48a64caea9c823fc8c1b119e8/gitdb-4.0.5-py3-none-any.whl (63kB)\n",
      "\u001B[K     |████████████████████████████████| 71kB 12.7MB/s \n",
      "\u001B[?25hCollecting smmap<4,>=3.0.1\n",
      "  Downloading https://files.pythonhosted.org/packages/b0/9a/4d409a6234eb940e6a78dfdfc66156e7522262f5f2fecca07dc55915952d/smmap-3.0.4-py2.py3-none-any.whl\n",
      "Building wheels for collected packages: pyyaml\n",
      "  Building wheel for pyyaml (setup.py) ... \u001B[?25l\u001B[?25hdone\n",
      "  Created wheel for pyyaml: filename=PyYAML-5.3.1-cp36-cp36m-linux_x86_64.whl size=44621 sha256=132f39d02b291cdc60b9eff7c14b051ee0f520f790ac3e3bdaf6823e9bf7fda3\n",
      "  Stored in directory: /root/.cache/pip/wheels/a7/c1/ea/cf5bd31012e735dc1dfea3131a2d5eae7978b251083d6247bd\n",
      "Successfully built pyyaml\n",
      "Installing collected packages: pyyaml, smmap, gitdb, gitpython, labml, einops, labml-helpers, labml-nn\n",
      "  Found existing installation: PyYAML 3.13\n",
      "    Uninstalling PyYAML-3.13:\n",
      "      Successfully uninstalled PyYAML-3.13\n",
      "Successfully installed einops-0.3.0 gitdb-4.0.5 gitpython-3.1.12 labml-0.4.89 labml-helpers-0.4.72 labml-nn-0.4.77 pyyaml-5.3.1 smmap-3.0.4\n"
     ]
    }
   ],
   "source": [
    "!pip install labml-nn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SE2VUQ6L5zxI"
   },
   "source": [
    "Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0hJXx_g0wS2C"
   },
   "outputs": [],
   "source": [
    "from labml import experiment\n",
    "from labml_nn.transformers.feedback.experiment import Configs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Lpggo0wM6qb-"
   },
   "source": [
    "Create an experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "bFcr9k-l4cAg"
   },
   "outputs": [],
   "source": [
    "experiment.create(name=\"feedback_transformer\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-OnHLi626tJt"
   },
   "source": [
    "Initialize configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Piz0c5f44hRo"
   },
   "outputs": [],
   "source": [
    "conf = Configs()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wwMzCqpD6vkL"
   },
   "source": [
    "Set experiment configurations and assign a configurations dictionary to override configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 17
    },
    "id": "e6hmQhTw4nks",
    "outputId": "91d99011-7a61-48fa-ee5c-2fa845883cec"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"overflow-x: scroll;\"></pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "experiment.configs(conf,\n",
    "                    {'tokenizer': 'character',\n",
    "                    'text': 'tiny_shakespeare',\n",
    "                    'optimizer.learning_rate': 1.0,\n",
    "                    'optimizer.optimizer': 'Noam',\n",
    "                    'prompt': 'It is',\n",
    "                    'prompt_separator': '',\n",
    "\n",
    "                    'model': 'feedback_transformer',\n",
    "\n",
    "                    'train_loader': 'shuffled_train_loader',\n",
    "                    'valid_loader': 'shuffled_valid_loader',\n",
    "\n",
    "                    'seq_len': 64,\n",
    "                    'epochs': 128,\n",
    "                    'batch_size': 80,\n",
    "                    'inner_iterations': 25})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EvI7MtgJ61w5"
   },
   "source": [
    "Set PyTorch models for loading and saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 255
    },
    "id": "GDlt7dp-5ALt",
    "outputId": "bee57c09-4e71-4329-debb-b23ed309a3ef"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"overflow-x: scroll;\">Prepare model...\n",
       "  Prepare n_tokens...\n",
       "    Prepare text...\n",
       "      Prepare tokenizer<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t3.28ms</span>\n",
       "      Download<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t223.64ms</span>\n",
       "      Load data<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t3.95ms</span>\n",
       "      Tokenize<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t31.15ms</span>\n",
       "      Build vocabulary<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t103.48ms</span>\n",
       "    Prepare text<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t381.54ms</span>\n",
       "  Prepare n_tokens<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t388.76ms</span>\n",
       "  Prepare device...\n",
       "    Prepare device_info<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t68.00ms</span>\n",
       "  Prepare device<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t70.80ms</span>\n",
       "Prepare model<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t10,879.84ms</span>\n",
       "</pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "experiment.add_pytorch_models({'model': conf.model})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KJZRf8527GxL"
   },
   "source": [
    "Start the experiment and run the training loop."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "aIAWo7Fw5DR8",
    "outputId": "bfb52e21-1913-4bd0-b67f-bef8a14f0f05"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"overflow-x: scroll;\">\n",
       "<strong><span style=\"text-decoration: underline\">feedback_transformer</span></strong>: <span style=\"color: #208FFB\">d8eb9416530a11eb8fb50242ac1c0002</span>\n",
       "\t[dirty]: <strong><span style=\"color: #DDB62B\">\"\"</span></strong>\n",
       "Initialize<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t1.30ms</span>\n",
       "Prepare validator...\n",
       "  Prepare valid_loader<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t70.14ms</span>\n",
       "<span style=\"color: #C5C1B4\"></span>\n",
       "<span style=\"color: #C5C1B4\">--------------------------------------------------</span><span style=\"color: #DDB62B\"><strong><span style=\"text-decoration: underline\"></span></strong></span>\n",
       "<span style=\"color: #DDB62B\"><strong><span style=\"text-decoration: underline\">LABML WARNING</span></strong></span>\n",
       "<span style=\"color: #DDB62B\"><strong><span style=\"text-decoration: underline\"></span></strong></span>LabML App Warning: <span style=\"color: #60C6C8\">empty_token: </span><strong>Please create a valid token at https://app.labml.ai.</strong>\n",
       "<strong>Click on the experiment link to monitor the experiment and add it to your experiments list.</strong><span style=\"color: #C5C1B4\"></span>\n",
       "<span style=\"color: #C5C1B4\">--------------------------------------------------</span>\n",
       "<span style=\"color: #208FFB\">Monitor experiment at </span><a href='https://app.labml.ai/run?uuid=d8eb9416530a11eb8fb50242ac1c0002' target='blank'>https://app.labml.ai/run?uuid=d8eb9416530a11eb8fb50242ac1c0002</a>\n",
       "Prepare validator<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t168.60ms</span>\n",
       "Prepare trainer...\n",
       "  Prepare train_loader<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t93.88ms</span>\n",
       "Prepare trainer<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t125.90ms</span>\n",
       "Prepare training_loop...\n",
       "  Prepare loop_count<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t35.06ms</span>\n",
       "Prepare training_loop<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t268.53ms</span>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong>?</strong><strong>a</strong><strong>M</strong><strong>r</strong><strong>P</strong><strong>a</strong><strong>D</strong><strong>P</strong><strong>Y</strong><strong>B</strong><strong>n</strong><strong>O</strong><strong>r</strong><strong>P</strong><strong>W</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>P</strong><strong>B</strong><strong>n</strong><strong>r</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong>?</strong><strong>a</strong><strong>M</strong><strong>r</strong><strong>P</strong><strong>a</strong><strong>D</strong><strong>P</strong><strong>Y</strong><strong>B</strong><strong>n</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong><strong>r</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong>?</strong><strong>a</strong><strong>M</strong><strong> </strong><strong>s</strong><strong>s</strong><strong>s</strong><strong>s</strong><strong>s</strong><strong>s</strong><strong>s</strong><strong>s</strong><strong>s</strong><strong>s</strong><strong>o</strong><strong>s</strong><strong>o</strong><strong>s</strong><strong>o</strong><strong>s</strong><strong>o</strong><strong>s</strong><strong>o</strong><strong>s</strong><strong>o</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong> </strong><strong>t</strong><strong> </strong><strong>t</strong><strong> </strong><strong>t</strong><strong> </strong><strong>t</strong><strong> </strong><strong>t</strong><strong> </strong><strong>t</strong><strong> </strong><strong>t</strong><strong> </strong><strong>t</strong><strong> </strong><strong>t</strong><strong> </strong><strong>t</strong><strong> </strong><strong>t</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>o</strong><strong>u</strong><strong>r</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>a</strong><strong>n</strong><strong>d</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<strong><span style=\"color: #DDB62B\">1,003,840:  </span></strong>Sample:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 2,532ms  </span>Train:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 447,670ms  </span>Valid:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 19,478ms  </span> accuracy.train: <span style=\"color: #C5C1B4\">0.239323</span> loss.train: <span style=\"color: #C5C1B4\"> 2.31443</span> accuracy.valid: <span style=\"color: #C5C1B4\">0.250179</span> loss.valid: <span style=\"color: #C5C1B4\"> 2.39115</span>  <span style=\"color: #208FFB\">415,476ms</span><span style=\"color: #D160C4\">  0:06m/ 14:39m  </span>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>a</strong><strong>t</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong>a</strong><strong>t</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>r</strong><strong>e</strong><strong>n</strong><strong>g</strong><strong>e</strong><strong>n</strong><strong>g</strong><strong> </strong><strong>t</strong><strong>h</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>a</strong><strong>r</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>h</strong><strong>e</strong><strong>a</strong><strong>l</strong><strong>l</strong><strong> </strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>h</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>m</strong><strong>e</strong><strong>a</strong><strong>r</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>u</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>u</strong><strong>l</strong><strong>d</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>d</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>o</strong><strong> </strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>a</strong><strong>v</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>a</strong><strong>t</strong><strong>h</strong><strong> </strong><strong>h</strong><strong>e</strong><strong>a</strong><strong>r</strong><strong> </strong><strong>h</strong><strong>i</strong><strong>s</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong> </strong><strong>m</strong><strong>a</strong><strong>y</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong>\n",
       "<strong><span style=\"color: #DDB62B\">2,007,680:  </span></strong>Sample:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 2,528ms  </span>Train:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 445,410ms  </span>Valid:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 19,593ms  </span> accuracy.train: <span style=\"color: #C5C1B4\">0.378694</span> loss.train: <span style=\"color: #C5C1B4\"> 1.96300</span> accuracy.valid: <span style=\"color: #C5C1B4\">0.367457</span> loss.valid: <span style=\"color: #C5C1B4\"> 2.02065</span>  <span style=\"color: #208FFB\">418,682ms</span><span style=\"color: #D160C4\">  0:13m/ 14:39m  </span>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>r</strong><strong>o</strong><strong>v</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>r</strong><strong>o</strong><strong>v</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>k</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>k</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>a</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>a</strong><strong>t</strong><strong> </strong><strong>h</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>a</strong><strong>t</strong><strong>h</strong><strong> </strong><strong>h</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>u</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>n</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>m</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong> </strong><strong>s</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>n</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>n</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>r</strong><strong> </strong><strong>t</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>r</strong><strong> </strong><strong>t</strong><strong>o</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>i</strong><strong>l</strong><strong>l</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>i</strong><strong>l</strong><strong>l</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>i</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>e</strong><strong>n</strong><strong> </strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>l</strong><strong>l</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>l</strong><strong>l</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>a</strong><strong>r</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>n</strong><strong>t</strong><strong>r</strong><strong>y</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>u</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>a</strong><strong>y</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>a</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>i</strong><strong>n</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>n</strong><strong>t</strong><strong>r</strong><strong>o</strong><strong>u</strong><strong>n</strong><strong>t</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>r</strong><strong>e</strong><strong>s</strong><strong>t</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>r</strong><strong>s</strong><strong>e</strong><strong>,</strong><strong></strong>\n",
       "<strong></strong><strong>A</strong><strong>n</strong><strong>d</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>r</strong><strong>t</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>r</strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>n</strong><strong>d</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>t</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>d</strong><strong>o</strong><strong>w</strong><strong>n</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>d</strong><strong>e</strong><strong>a</strong><strong>t</strong><strong>h</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>d</strong><strong>e</strong><strong>a</strong><strong>t</strong><strong>h</strong><strong> </strong><strong>o</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>u</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>u</strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>r</strong><strong>o</strong><strong>v</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>r</strong><strong>s</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>a</strong><strong>v</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>o</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>r</strong><strong>o</strong><strong>v</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>r</strong><strong>o</strong><strong>n</strong><strong>g</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>m</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>m</strong><strong>m</strong><strong>o</strong><strong>n</strong><strong> </strong><strong>o</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>n</strong><strong>t</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>n</strong><strong>t</strong><strong>e</strong><strong>r</strong>\n",
       "<strong><span style=\"color: #DDB62B\">3,011,520:  </span></strong>Sample:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 2,548ms  </span>Train:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 445,601ms  </span>Valid:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 19,525ms  </span> accuracy.train: <span style=\"color: #C5C1B4\">0.482339</span> loss.train: <span style=\"color: #C5C1B4\"> 1.35200</span> accuracy.valid: <span style=\"color: #C5C1B4\">0.451241</span> loss.valid: <span style=\"color: #C5C1B4\"> 1.79424</span>  <span style=\"color: #208FFB\">419,907ms</span><span style=\"color: #D160C4\">  0:20m/ 14:34m  </span>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>n</strong><strong>t</strong><strong> </strong><strong>t</strong><strong>o</strong><strong> </strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>o</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>n</strong><strong>o</strong><strong>t</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>r</strong><strong>e</strong><strong>a</strong><strong>t</strong><strong>h</strong><strong>,</strong><strong></strong>\n",
       "<strong></strong><strong>T</strong><strong>h</strong><strong>a</strong><strong>t</strong><strong> </strong><strong>w</strong><strong>h</strong><strong>e</strong><strong>n</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>m</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>o</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>m</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>o</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>n</strong><strong>t</strong><strong>e</strong><strong>r</strong><strong>s</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>o</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>e</strong><strong>r</strong><strong>p</strong><strong>e</strong><strong>r</strong><strong>e</strong><strong>t</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>e</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>m</strong><strong>m</strong><strong>o</strong><strong>n</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>r</strong><strong>t</strong><strong>e</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>n</strong><strong>t</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>e</strong><strong>a</strong><strong>c</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>n</strong><strong>t</strong><strong>r</strong><strong>y</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>n</strong><strong>t</strong><strong>r</strong><strong>e</strong><strong>a</strong><strong>t</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>n</strong><strong>t</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>n</strong><strong>t</strong><strong>e</strong><strong>r</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>n</strong><strong>t</strong><strong>r</strong><strong>y</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>n</strong><strong>t</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>n</strong><strong>o</strong><strong>t</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>d</strong><strong>a</strong><strong>y</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>d</strong><strong>a</strong><strong>y</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>m</strong><strong>a</strong><strong>n</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>m</strong><strong>o</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>n</strong><strong>t</strong><strong>e</strong><strong>r</strong><strong>n</strong><strong>e</strong><strong>s</strong><strong>s</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>l</strong><strong>o</strong><strong>r</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>r</strong><strong>i</strong><strong>n</strong><strong>c</strong><strong>e</strong><strong> </strong><strong>o</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>f</strong><strong>o</strong><strong>u</strong><strong>l</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>n</strong><strong>,</strong><strong></strong>\n",
       "<strong></strong><strong>A</strong><strong>n</strong><strong>d</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>r</strong><strong>e</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>d</strong><strong>a</strong><strong>y</strong><strong>s</strong><strong> </strong><strong>o</strong><strong>f</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>f</strong><strong>a</strong><strong>i</strong><strong>r</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>n</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>d</strong><strong>e</strong><strong>a</strong><strong>t</strong><strong>h</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>i</strong><strong>n</strong><strong>g</strong><strong> </strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>d</strong><strong>e</strong><strong>a</strong><strong>t</strong><strong>h</strong><strong>,</strong><strong></strong>\n",
       "<strong></strong><strong>A</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>a</strong><strong>t</strong><strong> </strong>\n",
       "<strong><span style=\"color: #DDB62B\">4,015,360:  </span></strong>Sample:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 2,574ms  </span>Train:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 445,980ms  </span>Valid:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 19,625ms  </span> accuracy.train: <span style=\"color: #C5C1B4\">0.532161</span> loss.train: <span style=\"color: #C5C1B4\"> 1.32791</span> accuracy.valid: <span style=\"color: #C5C1B4\">0.493120</span> loss.valid: <span style=\"color: #C5C1B4\"> 1.64298</span>  <span style=\"color: #208FFB\">420,841ms</span><span style=\"color: #D160C4\">  0:28m/ 14:29m  </span>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>r</strong><strong>o</strong><strong>n</strong><strong>g</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>a</strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>r</strong><strong>i</strong><strong>n</strong><strong>c</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>r</strong><strong>i</strong><strong>n</strong><strong>c</strong><strong>e</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>r</strong><strong>s</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>s</strong><strong>i</strong><strong>n</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>n</strong><strong>o</strong><strong>t</strong><strong> </strong><strong>s</strong><strong>o</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>i</strong><strong>m</strong><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>m</strong><strong>m</strong><strong>o</strong><strong>n</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>m</strong><strong>m</strong><strong>a</strong><strong>n</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>t</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>t</strong><strong>i</strong><strong>m</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>f</strong><strong>a</strong><strong>i</strong><strong>r</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>f</strong><strong>a</strong><strong>i</strong><strong>r</strong><strong>e</strong><strong>s</strong><strong>t</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>n</strong><strong>o</strong><strong>t</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>e</strong><strong>o</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>n</strong><strong>a</strong><strong>t</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>l</strong><strong>d</strong><strong>,</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>a</strong><strong> </strong><strong>m</strong><strong>a</strong><strong>n</strong><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>d</strong><strong>e</strong><strong>a</strong><strong>r</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>d</strong><strong>e</strong><strong>a</strong><strong>t</strong><strong>h</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>t</strong><strong>a</strong><strong>t</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>e</strong><strong>a</strong><strong>r</strong><strong>t</strong><strong>h</strong><strong>,</strong><strong></strong>\n",
       "<strong></strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>l</strong><strong>i</strong><strong>k</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>w</strong><strong>i</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>m</strong><strong>o</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>e</strong><strong>o</strong><strong>p</strong><strong>l</strong><strong>e</strong><strong>,</strong><strong></strong>\n",
       "<strong></strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>l</strong><strong>a</strong><strong>w</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>s</strong><strong>o</strong><strong>n</strong><strong>,</strong><strong></strong>\n",
       "<strong></strong><strong>A</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>s</strong><strong>o</strong><strong> </strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>m</strong><strong>o</strong><strong>r</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>m</strong><strong>m</strong><strong>o</strong><strong>n</strong><strong> </strong><strong>t</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>d</strong><strong>e</strong><strong>a</strong><strong>r</strong><strong>e</strong><strong>r</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>e</strong><strong>r</strong><strong>p</strong><strong>o</strong><strong>s</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>h</strong><strong>o</strong><strong>u</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>b</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>l</strong><strong>a</strong><strong>s</strong><strong>t</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>p</strong><strong>r</strong><strong>i</strong><strong>n</strong><strong>c</strong><strong>e</strong><strong> </strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>s</strong><strong>e</strong><strong>n</strong><strong>t</strong><strong>e</strong><strong>n</strong><strong>c</strong><strong>e</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>u</strong><strong>r</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>m</strong><strong>m</strong><strong>o</strong><strong>n</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>c</strong><strong>o</strong><strong>m</strong><strong>m</strong><strong>o</strong><strong>n</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>n</strong><strong>o</strong><strong>t</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>w</strong><strong>o</strong><strong>r</strong><strong>l</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>m</strong><strong>e</strong><strong>n</strong><strong> </strong><strong>a</strong><strong>n</strong><strong>d</strong>\n",
       "<span style=\"color: #C5C1B4\">It is</span><strong> </strong><strong>n</strong><strong>o</strong><strong>t</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>a</strong><strong>n</strong><strong>d</strong><strong> </strong><strong>o</strong><strong>f</strong><strong> </strong><strong>t</strong><strong>h</strong><strong>e</strong><strong> </strong><strong>h</strong><strong>o</strong><strong>u</strong><strong>s</strong>\n",
       "<strong><span style=\"color: #DDB62B\">4,931,840:  </span></strong>Sample:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 2,593ms  </span>Train:<span style=\"color: #C5C1B4\">  90%</span><span style=\"color: #208FFB\"> 464,871ms  </span>Valid:<span style=\"color: #C5C1B4\">  86%</span><span style=\"color: #208FFB\"> 12,860ms  </span> accuracy.train: <strong>0.555951</strong> loss.train: <strong> 1.44885</strong> accuracy.valid: <span style=\"color: #C5C1B4\">0.512315</span> loss.valid: <span style=\"color: #C5C1B4\"> 1.59942</span>  <span style=\"color: #208FFB\">420,841ms</span><span style=\"color: #D160C4\">  0:34m/ 14:23m  </span>\n",
       "Saving model...\n",
       "<span style=\"color: #E75C58\">Killing loop...</span>\n",
       "<strong><span style=\"color: #DDB62B\">Still updating LabML App, please wait for it to complete...</span></strong>\n",
       "<span style=\"color: #208FFB\">Updating App. Please wait...</span></pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "\u001B[0;32m<ipython-input-11-fb962b998049>\u001B[0m in \u001B[0;36m<module>\u001B[0;34m()\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[0;31m# Start the experiment\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      2\u001B[0m \u001B[0;32mwith\u001B[0m \u001B[0mexperiment\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mstart\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 3\u001B[0;31m     \u001B[0mconf\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m",
      "\u001B[0;32m/usr/local/lib/python3.6/dist-packages/labml_helpers/train_valid.py\u001B[0m in \u001B[0;36mrun\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m    238\u001B[0m         \u001B[0m_\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtrainer\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    239\u001B[0m         \u001B[0;32mfor\u001B[0m \u001B[0m_\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtraining_loop\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 240\u001B[0;31m             \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun_step\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    241\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    242\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0msample\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.6/dist-packages/labml_helpers/train_valid.py\u001B[0m in \u001B[0;36mrun_step\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m    226\u001B[0m             \u001B[0;32mwith\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mmode\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mupdate\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mis_train\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    227\u001B[0m                 \u001B[0;32mwith\u001B[0m \u001B[0mtracker\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mnamespace\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'train'\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 228\u001B[0;31m                     \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtrainer\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    229\u001B[0m             \u001B[0;32mif\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mvalidator\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    230\u001B[0m                 \u001B[0;32mwith\u001B[0m \u001B[0mtracker\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mnamespace\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'valid'\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.6/dist-packages/labml_helpers/train_valid.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m    130\u001B[0m                 \u001B[0msm\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mon_epoch_start\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    131\u001B[0m         \u001B[0;32mwith\u001B[0m \u001B[0mtorch\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mset_grad_enabled\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mmode\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mis_train\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 132\u001B[0;31m             \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m__iterate\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    133\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    134\u001B[0m         \u001B[0;32mif\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_batch_index\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcompleted\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.6/dist-packages/labml_helpers/train_valid.py\u001B[0m in \u001B[0;36m__iterate\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m    143\u001B[0m                 \u001B[0mbatch\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mnext\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m__iterable\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    144\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 145\u001B[0;31m                 \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mstep\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mbatch\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_batch_index\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    146\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    147\u001B[0m                 \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_batch_index\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mstep\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.6/dist-packages/labml_nn/experiments/nlp_autoregression.py\u001B[0m in \u001B[0;36mstep\u001B[0;34m(self, batch, batch_idx)\u001B[0m\n\u001B[1;32m     70\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     71\u001B[0m         \u001B[0;32mif\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mmode\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mis_train\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 72\u001B[0;31m             \u001B[0mloss\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mbackward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     73\u001B[0m             \u001B[0mtorch\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mnn\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mutils\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mclip_grad_norm_\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mmodel\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mparameters\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmax_norm\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;36m1.\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     74\u001B[0m             \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0moptimizer\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mstep\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.6/dist-packages/torch/tensor.py\u001B[0m in \u001B[0;36mbackward\u001B[0;34m(self, gradient, retain_graph, create_graph)\u001B[0m\n\u001B[1;32m    219\u001B[0m                 \u001B[0mretain_graph\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mretain_graph\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    220\u001B[0m                 create_graph=create_graph)\n\u001B[0;32m--> 221\u001B[0;31m         \u001B[0mtorch\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mautograd\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mbackward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mgradient\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mretain_graph\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mcreate_graph\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    222\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    223\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0mregister_hook\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mhook\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py\u001B[0m in \u001B[0;36mbackward\u001B[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001B[0m\n\u001B[1;32m    130\u001B[0m     Variable._execution_engine.run_backward(\n\u001B[1;32m    131\u001B[0m         \u001B[0mtensors\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mgrad_tensors_\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mretain_graph\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mcreate_graph\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 132\u001B[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001B[0m\u001B[1;32m    133\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    134\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.6/dist-packages/labml_helpers/training_loop.py\u001B[0m in \u001B[0;36m__handler\u001B[0;34m(self, sig, frame)\u001B[0m\n\u001B[1;32m    162\u001B[0m             \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m__finish\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    163\u001B[0m             \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mlog\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'Killing loop...'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mText\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdanger\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 164\u001B[0;31m             \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mold_handler\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0msig\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mframe\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    165\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    166\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0m__str__\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "source": [
    "# Start the experiment\n",
    "with experiment.start():\n",
    "    conf.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "oBXXlP2b7XZO"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Feedback Transformer",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}